﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Net;
using LingDong.HtmlParser;

namespace LingDong.NaiveCrawler
{
    public class Crawl
    {
        /// <summary>
        /// Download target catelogue url
        /// Parse it's links and save them to a file
        /// </summary>
        /// <param name="url"></param>
        /// <param name="file"></param>
        public static void DownloadCataloguePage(string url, string file)
        {
            DocumentWithLinks doc = new DocumentWithLinks(GetUrlContent(url));

            SaveLines(doc.References, file);
        }

        /// <summary>
        /// Download target catelogue url
        /// Parse it's links and save them to a file
        /// </summary>
        /// <param name="url">target catelogue url</param>
        /// <param name="file">target file to store links</param>
        /// <param name="startWith">select link with specific prefix</param>
        public static void DownloadCataloguePage(string url, string linkPrefix, string file)
        {
            DocumentWithLinks doc = new DocumentWithLinks(GetUrlContent(url), url);

            List<string> lines = doc.References.Where(s => s.StartsWith(linkPrefix)).Distinct().ToList();
            SaveLines(lines, file);
        }
        
        public static void DownloadCataloguePage(string url, string prefix, string innerContains, string file)
        {
            DocumentWithLinks doc = new DocumentWithLinks(GetUrlContent(url));
            List<string> links = doc.References.Where(s => s.StartsWith(prefix)).ToList();

            List<string> lines = new List<string>();
            foreach (string link in links)
            {
                string inner;
                if (doc.ReferenceInnerText.TryGetValue(link, out inner) &&
                    inner.Contains(innerContains))
                {
                    lines.Add(link);
                }
            }
            SaveLines(lines, file);
        }

        public static void DownloadUrlListFromFile(string outputFile, string outputDir)
        {
            // load url list
            List<string> urlList = new List<string>();
            using (StreamReader sr = new StreamReader(outputFile))
            {
                while (!sr.EndOfStream)
                {
                    urlList.Add(sr.ReadLine());
                }
            }

            string indexFile = outputDir + "index.txt";
            StreamWriter indexSW = new StreamWriter(indexFile, true);

            int count = urlList.Count;
            for (int i = 0; i < count; i++)
            {
                string html = GetUrlContent(urlList[i]);
                if (String.IsNullOrEmpty(html))
                    continue;

                string domain = urlList[i].Substring(11);
                domain = domain.Substring(0, domain.IndexOf('.'));
                string htmlFile = String.Format("{0}{1}_{2}.html", outputDir, domain, i);

                // write back
                using (StreamWriter sw = new StreamWriter(htmlFile, false, Encoding.UTF8))
                {
                    sw.WriteLine(html);
                }
                indexSW.WriteLine(String.Format("{0}\t{1}", urlList[i], htmlFile));
                indexSW.Flush();
                Console.WriteLine(urlList[i]);

                System.Threading.Thread.Sleep(1000);
            }

            indexSW.Close();
        }

        private static string GetUrlContent(string url)
        {
            try
            {
                WebClient client = new WebClient();
                client.Encoding = Encoding.GetEncoding("gb2312");
                //client.Encoding = Encoding.UTF8;
                return client.DownloadString(url);
            }
            catch (Exception)
            {
                return String.Empty;
            }
        }

        private static void SaveLines(List<string> lines, string file)
        {
            using (StreamWriter sw = new StreamWriter(file, true))
            {
                foreach (string link in lines)
                {
                    sw.WriteLine(link);
                }
            }
        }
    }
}
